{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\anaconda\\anaconda\\envs\\tensorflow\\lib\\site-packages\\sklearn\\utils\\fixes.py:313: FutureWarning: numpy not_equal will not check object identity in the future. The comparison did not return the same result as suggested by the identity (`is`)) and will change.\n",
      "  _nan_object_mask = _nan_object_array != _nan_object_array\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.layers import Input, Dense, Conv1D, Flatten, MaxPooling1D, Conv2D, MaxPooling2D, AveragePooling2D, Dropout, Reshape, normalization\n",
    "from keras.models import Model\n",
    "from keras.utils import to_categorical\n",
    "import keras.backend as K\n",
    "from keras.layers.recurrent import LSTM\n",
    "from sklearn import metrics\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def precision(y_true, y_pred):\n",
    "    # Calculates the precision\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "def recall(y_true, y_pred):\n",
    "    # Calculates the recall\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    recall = true_positives / (possible_positives + K.epsilon())\n",
    "    return recall\n",
    "\n",
    "def f1(test_Y, pre_test_y):\n",
    "    \"\"\"F1-score\"\"\"\n",
    "    Precision = precision(test_Y, pre_test_y)\n",
    "    Recall = recall(test_Y, pre_test_y)\n",
    "    f1 = 2 * ((Precision * Recall) / (Precision + Recall + K.epsilon()))\n",
    "    return f1 \n",
    "\n",
    "def TP(test_Y,pre_test_y):\n",
    "    TP = K.sum(K.round(K.clip(test_Y * pre_test_y, 0, 1)))#TP\n",
    "    return TP\n",
    "\n",
    "def FN(test_Y,pre_test_y):\n",
    "    TP = K.sum(K.round(K.clip(test_Y * pre_test_y, 0, 1)))#TP\n",
    "    P=K.sum(K.round(K.clip(test_Y, 0, 1)))\n",
    "    FN = P-TP #FN=P-TP\n",
    "    return FN\n",
    "\n",
    "def TN(test_Y,pre_test_y):\n",
    "    TN=K.sum(K.round(K.clip((test_Y-K.ones_like(test_Y))*(pre_test_y-K.ones_like(pre_test_y)), 0, 1)))#TN\n",
    "    return TN\n",
    "\n",
    "def FP(test_Y,pre_test_y):\n",
    "    N = (-1)*K.sum(K.round(K.clip(test_Y-K.ones_like(test_Y), -1, 0)))#N\n",
    "    TN=K.sum(K.round(K.clip((test_Y-K.ones_like(test_Y))*(pre_test_y-K.ones_like(pre_test_y)), 0, 1)))#TN\n",
    "    FP=N-TN\n",
    "    return FP\n",
    "\n",
    "def dnn_model(train_X, train_Y, test_X, test_Y, lr, epoch, batch_size):\n",
    "    train_X = np.expand_dims(train_X, 2)\n",
    "    test_X = np.expand_dims(test_X, 2)\n",
    "    inputs = Input(shape = (train_X.shape[1], train_X.shape[2]))\n",
    "    x = Conv1D(32, kernel_size = 3, strides = 1, padding = 'valid', activation = 'relu')(inputs)\n",
    "    x = MaxPooling1D(pool_size = 2, strides = 2, padding = 'same')(x)\n",
    "    x = Flatten()(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(32, activation = 'relu')(x)\n",
    "    x = Dense(16, activation = 'relu')(x)\n",
    "    x = Dense(8, activation = 'relu')(x)\n",
    "    predictions = Dense(1, activation = 'sigmoid')(x)\n",
    "    model = Model(inputs = inputs, outputs = predictions)\n",
    "    print(\"model\")\n",
    "    model.compile(optimizer = 'RMSprop',\n",
    "                  loss = 'mean_squared_error',\n",
    "                  metrics = ['acc',precision,recall,f1,TP,FN,TN,FP])\n",
    "    print(\"compile\")\n",
    "    model.fit(train_X, train_Y, epochs = epoch, batch_size = 32, validation_data = (test_X, test_Y), shuffle = True)\n",
    "    model.save('CNN_model.h5')\n",
    "    pre_test_y = model.predict(test_X, batch_size = 50)\n",
    "    pre_train_y = model.predict(train_X, batch_size = 50)\n",
    "    test_auc = metrics.roc_auc_score(test_Y, pre_test_y)\n",
    "    train_auc = metrics.roc_auc_score(train_Y, pre_train_y)\n",
    "    print(\"train_auc: \", train_auc)\n",
    "    print(\"test_auc: \", test_auc) \n",
    "    return test_auc\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.387706    0.235774    0.0419513  ...,  0.0660476  -0.122586    0.0697366 ]\n",
      " [ 0.0954705   0.361979   -0.0432098  ..., -0.12312    -0.131958    0.0933789 ]\n",
      " [ 0.0558099   0.299304   -0.0562219  ..., -0.0769578  -0.114194    0.0707509 ]\n",
      " ..., \n",
      " [ 0.234206   -0.0604085  -0.0320356  ...,  0.221071    0.0604408  -0.052408  ]\n",
      " [ 0.168922    0.115964   -0.0967158  ..., -0.00971257  0.144195   -0.0897304 ]\n",
      " [ 0.127001    0.0762867   0.0180975  ..., -0.287323    0.149178   -0.124711  ]]\n",
      "X.shape:  (40, 200)\n",
      "Y.shape:  (40,)\n",
      "\n",
      "\n",
      "i:  0\n",
      "model\n",
      "compile\n",
      "Train on 26 samples, validate on 14 samples\n",
      "Epoch 1/20\n",
      "26/26 [==============================] - 1s 20ms/step - loss: 0.2477 - acc: 0.6154 - precision: 0.5556 - recall: 0.8333 - f1: 0.6667 - TP: 10.0000 - FN: 2.0000 - TN: 6.0000 - FP: 8.0000 - val_loss: 0.2237 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 2/20\n",
      "26/26 [==============================] - 0s 306us/step - loss: 0.2211 - acc: 0.7692 - precision: 0.8000 - recall: 0.6667 - f1: 0.7273 - TP: 8.0000 - FN: 4.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.2055 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 3/20\n",
      "26/26 [==============================] - 0s 383us/step - loss: 0.1988 - acc: 0.8077 - precision: 0.7692 - recall: 0.8333 - f1: 0.8000 - TP: 10.0000 - FN: 2.0000 - TN: 11.0000 - FP: 3.0000 - val_loss: 0.1942 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 4/20\n",
      "26/26 [==============================] - 0s 537us/step - loss: 0.1891 - acc: 0.7308 - precision: 0.7778 - recall: 0.5833 - f1: 0.6667 - TP: 7.0000 - FN: 5.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1827 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 5/20\n",
      "26/26 [==============================] - 0s 614us/step - loss: 0.1608 - acc: 0.8077 - precision: 0.8182 - recall: 0.7500 - f1: 0.7826 - TP: 9.0000 - FN: 3.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1749 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 6/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1499 - acc: 0.8077 - precision: 0.8182 - recall: 0.7500 - f1: 0.7826 - TP: 9.0000 - FN: 3.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1700 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 7/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1436 - acc: 0.8462 - precision: 0.8333 - recall: 0.8333 - f1: 0.8333 - TP: 10.0000 - FN: 2.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1686 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 8/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1332 - acc: 0.8462 - precision: 0.9000 - recall: 0.7500 - f1: 0.8182 - TP: 9.0000 - FN: 3.0000 - TN: 13.0000 - FP: 1.0000 - val_loss: 0.1648 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 9/20\n",
      "26/26 [==============================] - 0s 537us/step - loss: 0.1286 - acc: 0.8077 - precision: 0.8182 - recall: 0.7500 - f1: 0.7826 - TP: 9.0000 - FN: 3.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1631 - val_acc: 0.6429 - val_precision: 0.7143 - val_recall: 0.6250 - val_f1: 0.6667 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 10/20\n",
      "26/26 [==============================] - 0s 536us/step - loss: 0.1175 - acc: 0.8846 - precision: 0.8462 - recall: 0.9167 - f1: 0.8800 - TP: 11.0000 - FN: 1.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1700 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 11/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1175 - acc: 0.8846 - precision: 1.0000 - recall: 0.7500 - f1: 0.8571 - TP: 9.0000 - FN: 3.0000 - TN: 14.0000 - FP: 0.0000e+00 - val_loss: 0.1590 - val_acc: 0.6429 - val_precision: 0.7143 - val_recall: 0.6250 - val_f1: 0.6667 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 12/20\n",
      "26/26 [==============================] - 0s 537us/step - loss: 0.1225 - acc: 0.8846 - precision: 0.8462 - recall: 0.9167 - f1: 0.8800 - TP: 11.0000 - FN: 1.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1802 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 13/20\n",
      "26/26 [==============================] - 0s 614us/step - loss: 0.1144 - acc: 0.8846 - precision: 1.0000 - recall: 0.7500 - f1: 0.8571 - TP: 9.0000 - FN: 3.0000 - TN: 14.0000 - FP: 0.0000e+00 - val_loss: 0.1594 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 14/20\n",
      "26/26 [==============================] - 0s 499us/step - loss: 0.1262 - acc: 0.7692 - precision: 0.6875 - recall: 0.9167 - f1: 0.7857 - TP: 11.0000 - FN: 1.0000 - TN: 9.0000 - FP: 5.0000 - val_loss: 0.1918 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 15/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1183 - acc: 0.8846 - precision: 1.0000 - recall: 0.7500 - f1: 0.8571 - TP: 9.0000 - FN: 3.0000 - TN: 14.0000 - FP: 0.0000e+00 - val_loss: 0.1528 - val_acc: 0.7857 - val_precision: 0.7778 - val_recall: 0.8750 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 16/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1113 - acc: 0.8462 - precision: 0.7857 - recall: 0.9167 - f1: 0.8462 - TP: 11.0000 - FN: 1.0000 - TN: 11.0000 - FP: 3.0000 - val_loss: 0.1720 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 17/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.1060 - acc: 0.8846 - precision: 1.0000 - recall: 0.7500 - f1: 0.8571 - TP: 9.0000 - FN: 3.0000 - TN: 14.0000 - FP: 0.0000e+00 - val_loss: 0.1525 - val_acc: 0.7143 - val_precision: 0.8333 - val_recall: 0.6250 - val_f1: 0.7143 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 5.0000 - val_FP: 1.0000\n",
      "Epoch 18/20\n",
      "26/26 [==============================] - 0s 537us/step - loss: 0.1012 - acc: 0.9231 - precision: 0.9167 - recall: 0.9167 - f1: 0.9167 - TP: 11.0000 - FN: 1.0000 - TN: 13.0000 - FP: 1.0000 - val_loss: 0.1698 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 19/20\n",
      "26/26 [==============================] - 0s 575us/step - loss: 0.0968 - acc: 0.8846 - precision: 1.0000 - recall: 0.7500 - f1: 0.8571 - TP: 9.0000 - FN: 3.0000 - TN: 14.0000 - FP: 0.0000e+00 - val_loss: 0.1562 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 20/20\n",
      "26/26 [==============================] - 0s 461us/step - loss: 0.0842 - acc: 0.9615 - precision: 1.0000 - recall: 0.9167 - f1: 0.9565 - TP: 11.0000 - FN: 1.0000 - TN: 14.0000 - FP: 0.0000e+00 - val_loss: 0.1655 - val_acc: 0.7857 - val_precision: 1.0000 - val_recall: 0.6250 - val_f1: 0.7692 - val_TP: 5.0000 - val_FN: 3.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "train_auc:  0.964285714286\n",
      "test_auc:  0.875\n",
      "\n",
      "\n",
      "i:  1\n",
      "model\n",
      "compile\n",
      "Train on 27 samples, validate on 13 samples\n",
      "Epoch 1/20\n",
      "27/27 [==============================] - 1s 24ms/step - loss: 0.2504 - acc: 0.4815 - precision: 0.4815 - recall: 1.0000 - f1: 0.6500 - TP: 13.0000 - FN: 0.0000e+00 - TN: 0.0000e+00 - FP: 14.0000 - val_loss: 0.2431 - val_acc: 0.6154 - val_precision: 0.5833 - val_recall: 1.0000 - val_f1: 0.7368 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 1.0000 - val_FP: 5.0000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/20\n",
      "27/27 [==============================] - 0s 480us/step - loss: 0.2485 - acc: 0.4074 - precision: 0.4400 - recall: 0.8462 - f1: 0.5789 - TP: 11.0000 - FN: 2.0000 - TN: 0.0000e+00 - FP: 14.0000 - val_loss: 0.2283 - val_acc: 0.7692 - val_precision: 0.7500 - val_recall: 0.8571 - val_f1: 0.8000 - val_TP: 6.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 2.0000\n",
      "Epoch 3/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.2429 - acc: 0.5926 - precision: 0.5556 - recall: 0.7692 - f1: 0.6452 - TP: 10.0000 - FN: 3.0000 - TN: 6.0000 - FP: 8.0000 - val_loss: 0.2146 - val_acc: 0.7692 - val_precision: 0.7000 - val_recall: 1.0000 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 3.0000 - val_FP: 3.0000\n",
      "Epoch 4/20\n",
      "27/27 [==============================] - 0s 554us/step - loss: 0.2344 - acc: 0.7037 - precision: 0.6190 - recall: 1.0000 - f1: 0.7647 - TP: 13.0000 - FN: 0.0000e+00 - TN: 6.0000 - FP: 8.0000 - val_loss: 0.2083 - val_acc: 0.9231 - val_precision: 1.0000 - val_recall: 0.8571 - val_f1: 0.9231 - val_TP: 6.0000 - val_FN: 1.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 5/20\n",
      "27/27 [==============================] - 0s 591us/step - loss: 0.2293 - acc: 0.6296 - precision: 0.6000 - recall: 0.6923 - f1: 0.6429 - TP: 9.0000 - FN: 4.0000 - TN: 8.0000 - FP: 6.0000 - val_loss: 0.1961 - val_acc: 0.7692 - val_precision: 0.7000 - val_recall: 1.0000 - val_f1: 0.8235 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 3.0000 - val_FP: 3.0000\n",
      "Epoch 6/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.2269 - acc: 0.5556 - precision: 0.5200 - recall: 1.0000 - f1: 0.6842 - TP: 13.0000 - FN: 0.0000e+00 - TN: 2.0000 - FP: 12.0000 - val_loss: 0.1982 - val_acc: 0.9231 - val_precision: 1.0000 - val_recall: 0.8571 - val_f1: 0.9231 - val_TP: 6.0000 - val_FN: 1.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 7/20\n",
      "27/27 [==============================] - 0s 332us/step - loss: 0.2241 - acc: 0.7778 - precision: 0.7333 - recall: 0.8462 - f1: 0.7857 - TP: 11.0000 - FN: 2.0000 - TN: 10.0000 - FP: 4.0000 - val_loss: 0.1822 - val_acc: 0.9231 - val_precision: 0.8750 - val_recall: 1.0000 - val_f1: 0.9333 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 5.0000 - val_FP: 1.0000\n",
      "Epoch 8/20\n",
      "27/27 [==============================] - 0s 480us/step - loss: 0.2135 - acc: 0.7037 - precision: 0.6667 - recall: 0.7692 - f1: 0.7143 - TP: 10.0000 - FN: 3.0000 - TN: 9.0000 - FP: 5.0000 - val_loss: 0.1756 - val_acc: 0.9231 - val_precision: 0.8750 - val_recall: 1.0000 - val_f1: 0.9333 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 5.0000 - val_FP: 1.0000\n",
      "Epoch 9/20\n",
      "27/27 [==============================] - 0s 443us/step - loss: 0.2101 - acc: 0.7407 - precision: 0.6875 - recall: 0.8462 - f1: 0.7586 - TP: 11.0000 - FN: 2.0000 - TN: 9.0000 - FP: 5.0000 - val_loss: 0.1787 - val_acc: 0.8462 - val_precision: 1.0000 - val_recall: 0.7143 - val_f1: 0.8333 - val_TP: 5.0000 - val_FN: 2.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 10/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.2082 - acc: 0.7407 - precision: 0.7500 - recall: 0.6923 - f1: 0.7200 - TP: 9.0000 - FN: 4.0000 - TN: 11.0000 - FP: 3.0000 - val_loss: 0.1611 - val_acc: 1.0000 - val_precision: 1.0000 - val_recall: 1.0000 - val_f1: 1.0000 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 11/20\n",
      "27/27 [==============================] - 0s 554us/step - loss: 0.1988 - acc: 0.7407 - precision: 0.6875 - recall: 0.8462 - f1: 0.7586 - TP: 11.0000 - FN: 2.0000 - TN: 9.0000 - FP: 5.0000 - val_loss: 0.1645 - val_acc: 0.9231 - val_precision: 1.0000 - val_recall: 0.8571 - val_f1: 0.9231 - val_TP: 6.0000 - val_FN: 1.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 12/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.1945 - acc: 0.8148 - precision: 0.8333 - recall: 0.7692 - f1: 0.8000 - TP: 10.0000 - FN: 3.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1501 - val_acc: 0.9231 - val_precision: 0.8750 - val_recall: 1.0000 - val_f1: 0.9333 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 5.0000 - val_FP: 1.0000\n",
      "Epoch 13/20\n",
      "27/27 [==============================] - 0s 407us/step - loss: 0.1900 - acc: 0.7778 - precision: 0.7333 - recall: 0.8462 - f1: 0.7857 - TP: 11.0000 - FN: 2.0000 - TN: 10.0000 - FP: 4.0000 - val_loss: 0.1630 - val_acc: 0.8462 - val_precision: 1.0000 - val_recall: 0.7143 - val_f1: 0.8333 - val_TP: 5.0000 - val_FN: 2.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 14/20\n",
      "27/27 [==============================] - 0s 332us/step - loss: 0.1855 - acc: 0.7778 - precision: 0.8182 - recall: 0.6923 - f1: 0.7500 - TP: 9.0000 - FN: 4.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1386 - val_acc: 1.0000 - val_precision: 1.0000 - val_recall: 1.0000 - val_f1: 1.0000 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 15/20\n",
      "27/27 [==============================] - 0s 406us/step - loss: 0.1845 - acc: 0.8148 - precision: 0.7857 - recall: 0.8462 - f1: 0.8148 - TP: 11.0000 - FN: 2.0000 - TN: 11.0000 - FP: 3.0000 - val_loss: 0.1474 - val_acc: 0.8462 - val_precision: 1.0000 - val_recall: 0.7143 - val_f1: 0.8333 - val_TP: 5.0000 - val_FN: 2.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 16/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.1761 - acc: 0.8148 - precision: 0.8333 - recall: 0.7692 - f1: 0.8000 - TP: 10.0000 - FN: 3.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1304 - val_acc: 1.0000 - val_precision: 1.0000 - val_recall: 1.0000 - val_f1: 1.0000 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 17/20\n",
      "27/27 [==============================] - 0s 591us/step - loss: 0.1731 - acc: 0.8148 - precision: 0.7857 - recall: 0.8462 - f1: 0.8148 - TP: 11.0000 - FN: 2.0000 - TN: 11.0000 - FP: 3.0000 - val_loss: 0.1382 - val_acc: 0.8462 - val_precision: 1.0000 - val_recall: 0.7143 - val_f1: 0.8333 - val_TP: 5.0000 - val_FN: 2.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 18/20\n",
      "27/27 [==============================] - 0s 591us/step - loss: 0.1643 - acc: 0.7778 - precision: 0.8182 - recall: 0.6923 - f1: 0.7500 - TP: 9.0000 - FN: 4.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1178 - val_acc: 1.0000 - val_precision: 1.0000 - val_recall: 1.0000 - val_f1: 1.0000 - val_TP: 7.0000 - val_FN: 0.0000e+00 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 19/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.1660 - acc: 0.7778 - precision: 0.7333 - recall: 0.8462 - f1: 0.7857 - TP: 11.0000 - FN: 2.0000 - TN: 10.0000 - FP: 4.0000 - val_loss: 0.1260 - val_acc: 0.9231 - val_precision: 1.0000 - val_recall: 0.8571 - val_f1: 0.9231 - val_TP: 6.0000 - val_FN: 1.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "Epoch 20/20\n",
      "27/27 [==============================] - 0s 554us/step - loss: 0.1609 - acc: 0.8148 - precision: 0.8333 - recall: 0.7692 - f1: 0.8000 - TP: 10.0000 - FN: 3.0000 - TN: 12.0000 - FP: 2.0000 - val_loss: 0.1103 - val_acc: 0.9231 - val_precision: 1.0000 - val_recall: 0.8571 - val_f1: 0.9231 - val_TP: 6.0000 - val_FN: 1.0000 - val_TN: 6.0000 - val_FP: 0.0000e+00\n",
      "train_auc:  0.912087912088\n",
      "test_auc:  1.0\n",
      "\n",
      "\n",
      "i:  2\n",
      "model\n",
      "compile\n",
      "Train on 27 samples, validate on 13 samples\n",
      "Epoch 1/20\n",
      "27/27 [==============================] - 1s 23ms/step - loss: 0.2494 - acc: 0.5556 - precision: 0.5714 - recall: 0.8000 - f1: 0.6667 - TP: 12.0000 - FN: 3.0000 - TN: 1.0000 - FP: 11.0000 - val_loss: 0.2494 - val_acc: 0.3846 - val_precision: 0.3846 - val_recall: 1.0000 - val_f1: 0.5556 - val_TP: 5.0000 - val_FN: 0.0000e+00 - val_TN: 0.0000e+00 - val_FP: 8.0000\n",
      "Epoch 2/20\n",
      "27/27 [==============================] - 0s 333us/step - loss: 0.2296 - acc: 0.5556 - precision: 0.5556 - recall: 1.0000 - f1: 0.7143 - TP: 15.0000 - FN: 0.0000e+00 - TN: 0.0000e+00 - FP: 12.0000 - val_loss: 0.2794 - val_acc: 0.3846 - val_precision: 0.3846 - val_recall: 1.0000 - val_f1: 0.5556 - val_TP: 5.0000 - val_FN: 0.0000e+00 - val_TN: 0.0000e+00 - val_FP: 8.0000\n",
      "Epoch 3/20\n",
      "27/27 [==============================] - 0s 443us/step - loss: 0.2163 - acc: 0.5556 - precision: 0.5556 - recall: 1.0000 - f1: 0.7143 - TP: 15.0000 - FN: 0.0000e+00 - TN: 0.0000e+00 - FP: 12.0000 - val_loss: 0.2487 - val_acc: 0.3846 - val_precision: 0.3846 - val_recall: 1.0000 - val_f1: 0.5556 - val_TP: 5.0000 - val_FN: 0.0000e+00 - val_TN: 0.0000e+00 - val_FP: 8.0000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.2033 - acc: 0.5556 - precision: 0.5556 - recall: 1.0000 - f1: 0.7143 - TP: 15.0000 - FN: 0.0000e+00 - TN: 0.0000e+00 - FP: 12.0000 - val_loss: 0.2587 - val_acc: 0.3846 - val_precision: 0.3846 - val_recall: 1.0000 - val_f1: 0.5556 - val_TP: 5.0000 - val_FN: 0.0000e+00 - val_TN: 0.0000e+00 - val_FP: 8.0000\n",
      "Epoch 5/20\n",
      "27/27 [==============================] - 0s 370us/step - loss: 0.1837 - acc: 0.5556 - precision: 0.5556 - recall: 1.0000 - f1: 0.7143 - TP: 15.0000 - FN: 0.0000e+00 - TN: 0.0000e+00 - FP: 12.0000 - val_loss: 0.2543 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 6/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.1821 - acc: 0.7778 - precision: 0.7368 - recall: 0.9333 - f1: 0.8235 - TP: 14.0000 - FN: 1.0000 - TN: 7.0000 - FP: 5.0000 - val_loss: 0.2523 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 7/20\n",
      "27/27 [==============================] - 0s 369us/step - loss: 0.1743 - acc: 0.8148 - precision: 0.7778 - recall: 0.9333 - f1: 0.8485 - TP: 14.0000 - FN: 1.0000 - TN: 8.0000 - FP: 4.0000 - val_loss: 0.2492 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 8/20\n",
      "27/27 [==============================] - 0s 591us/step - loss: 0.1728 - acc: 0.8148 - precision: 0.7778 - recall: 0.9333 - f1: 0.8485 - TP: 14.0000 - FN: 1.0000 - TN: 8.0000 - FP: 4.0000 - val_loss: 0.2455 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 9/20\n",
      "27/27 [==============================] - 0s 591us/step - loss: 0.1685 - acc: 0.8519 - precision: 0.8235 - recall: 0.9333 - f1: 0.8750 - TP: 14.0000 - FN: 1.0000 - TN: 9.0000 - FP: 3.0000 - val_loss: 0.2435 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 10/20\n",
      "27/27 [==============================] - 0s 369us/step - loss: 0.1690 - acc: 0.7407 - precision: 0.7000 - recall: 0.9333 - f1: 0.8000 - TP: 14.0000 - FN: 1.0000 - TN: 6.0000 - FP: 6.0000 - val_loss: 0.2379 - val_acc: 0.7692 - val_precision: 0.6667 - val_recall: 0.8000 - val_f1: 0.7273 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 6.0000 - val_FP: 2.0000\n",
      "Epoch 11/20\n",
      "27/27 [==============================] - 0s 480us/step - loss: 0.1679 - acc: 0.8889 - precision: 0.8750 - recall: 0.9333 - f1: 0.9032 - TP: 14.0000 - FN: 1.0000 - TN: 10.0000 - FP: 2.0000 - val_loss: 0.2463 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 12/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.1628 - acc: 0.7407 - precision: 0.7000 - recall: 0.9333 - f1: 0.8000 - TP: 14.0000 - FN: 1.0000 - TN: 6.0000 - FP: 6.0000 - val_loss: 0.2358 - val_acc: 0.7692 - val_precision: 0.7500 - val_recall: 0.6000 - val_f1: 0.6667 - val_TP: 3.0000 - val_FN: 2.0000 - val_TN: 7.0000 - val_FP: 1.0000\n",
      "Epoch 13/20\n",
      "27/27 [==============================] - 0s 480us/step - loss: 0.1677 - acc: 0.8519 - precision: 0.8667 - recall: 0.8667 - f1: 0.8667 - TP: 13.0000 - FN: 2.0000 - TN: 10.0000 - FP: 2.0000 - val_loss: 0.2337 - val_acc: 0.6923 - val_precision: 0.5714 - val_recall: 0.8000 - val_f1: 0.6667 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 5.0000 - val_FP: 3.0000\n",
      "Epoch 14/20\n",
      "27/27 [==============================] - 0s 517us/step - loss: 0.1570 - acc: 0.8519 - precision: 0.8235 - recall: 0.9333 - f1: 0.8750 - TP: 14.0000 - FN: 1.0000 - TN: 9.0000 - FP: 3.0000 - val_loss: 0.2320 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 15/20\n",
      "27/27 [==============================] - 0s 443us/step - loss: 0.1556 - acc: 0.8519 - precision: 0.8235 - recall: 0.9333 - f1: 0.8750 - TP: 14.0000 - FN: 1.0000 - TN: 9.0000 - FP: 3.0000 - val_loss: 0.2326 - val_acc: 0.6154 - val_precision: 0.5000 - val_recall: 0.8000 - val_f1: 0.6154 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 4.0000 - val_FP: 4.0000\n",
      "Epoch 16/20\n",
      "27/27 [==============================] - 0s 554us/step - loss: 0.1601 - acc: 0.8148 - precision: 0.7778 - recall: 0.9333 - f1: 0.8485 - TP: 14.0000 - FN: 1.0000 - TN: 8.0000 - FP: 4.0000 - val_loss: 0.2227 - val_acc: 0.8462 - val_precision: 0.8000 - val_recall: 0.8000 - val_f1: 0.8000 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 7.0000 - val_FP: 1.0000\n",
      "Epoch 17/20\n",
      "27/27 [==============================] - 0s 518us/step - loss: 0.1590 - acc: 0.9259 - precision: 0.9333 - recall: 0.9333 - f1: 0.9333 - TP: 14.0000 - FN: 1.0000 - TN: 11.0000 - FP: 1.0000 - val_loss: 0.2181 - val_acc: 0.6923 - val_precision: 0.5714 - val_recall: 0.8000 - val_f1: 0.6667 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 5.0000 - val_FP: 3.0000\n",
      "Epoch 18/20\n",
      "27/27 [==============================] - 0s 369us/step - loss: 0.1488 - acc: 0.8519 - precision: 0.8235 - recall: 0.9333 - f1: 0.8750 - TP: 14.0000 - FN: 1.0000 - TN: 9.0000 - FP: 3.0000 - val_loss: 0.2141 - val_acc: 0.6923 - val_precision: 0.5714 - val_recall: 0.8000 - val_f1: 0.6667 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 5.0000 - val_FP: 3.0000\n",
      "Epoch 19/20\n",
      "27/27 [==============================] - 0s 406us/step - loss: 0.1531 - acc: 0.7778 - precision: 0.7368 - recall: 0.9333 - f1: 0.8235 - TP: 14.0000 - FN: 1.0000 - TN: 7.0000 - FP: 5.0000 - val_loss: 0.2118 - val_acc: 0.8462 - val_precision: 0.8000 - val_recall: 0.8000 - val_f1: 0.8000 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 7.0000 - val_FP: 1.0000\n",
      "Epoch 20/20\n",
      "27/27 [==============================] - 0s 443us/step - loss: 0.1554 - acc: 0.8889 - precision: 0.9286 - recall: 0.8667 - f1: 0.8966 - TP: 13.0000 - FN: 2.0000 - TN: 11.0000 - FP: 1.0000 - val_loss: 0.2144 - val_acc: 0.6923 - val_precision: 0.5714 - val_recall: 0.8000 - val_f1: 0.6667 - val_TP: 4.0000 - val_FN: 1.0000 - val_TN: 5.0000 - val_FP: 3.0000\n",
      "train_auc:  0.952777777778\n",
      "test_auc:  0.8375\n"
     ]
    }
   ],
   "source": [
    "\n",
    "data = np.array(pd.read_csv(\"3_train_vecs.csv\"))\n",
    "pos_number = 20 # NOTE: the number of postive sample in train file\n",
    "#CNN_model = 'CNN_model.h5'\n",
    "\n",
    "X1 = data[0:pos_number, 1:]\n",
    "Y1 = data[0:pos_number, 0]\n",
    "X2 = data[pos_number:, 1:]\n",
    "Y2 = data[pos_number:, 0]\n",
    "X = np.concatenate([X1, X2], 0)\n",
    "Y = np.concatenate([Y1, Y2], 0)\n",
    "#Y = Y.reshape((Y.shape[0], -1))\n",
    "print (X)\n",
    "print (\"X.shape: \", X.shape)\n",
    "print (\"Y.shape: \", Y.shape)\n",
    "\n",
    "lr = 0.4\n",
    "epoch = 20\n",
    "batch_size = 32\n",
    "kf = KFold(n_splits = 3, shuffle = True, random_state = 42)\n",
    "#kf = KFold(n_splits = 5, shuffle = False)\n",
    "kf = kf.split(X)\n",
    "\n",
    "test_aucs = []\n",
    "for i, (train_fold, validate_fold) in enumerate(kf):\n",
    "    print(\"\\n\\ni: \", i)\n",
    "    test_auc = dnn_model(X[train_fold], Y[train_fold], X[validate_fold], Y[validate_fold], lr, epoch, batch_size)\n",
    "    test_aucs.append(test_auc)\n",
    "w = open(\"train_Result.txt\", \"w\")\n",
    "for j in test_aucs: \n",
    "    w.write(str(j) + ',')\n",
    "w.write('\\n')\n",
    "w.write(str(np.mean(test_aucs)) + '\\n')\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
